<!DOCTYPE html>
<html>
<head>
	<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
	<meta name="generator" content="ApiGen 2.8.0" />
	<meta name="robots" content="noindex" />

	<title>File Crawler.php</title>

	<script type="text/javascript" src="resources/combined.js?174271566"></script>
	<script type="text/javascript" src="elementlist.js?2473211788"></script>
	<link rel="stylesheet" type="text/css" media="all" href="resources/style.css?3505392360" />

</head>

<body>
<div id="left">
	<div id="menu">
		<a href="index.html" title="Overview"><span>Overview</span></a>


		<div id="groups">
		</div>



		<div id="elements">
			<h3>Classes</h3>
			<ul>
				<li><a href="class-Indexer.html">Indexer</a></li>
				<li><a href="class-SBWord.html">SBWord</a></li>
				<li class="active"><a href="class-SpiderBot.html">SpiderBot</a></li>
			</ul>





		</div>
	</div>
</div>

<div id="splitter"></div>

<div id="right">
<div id="rightInner">
	<form id="search">
		<input type="hidden" name="cx" value="" />
		<input type="hidden" name="ie" value="UTF-8" />
		<input type="text" name="q" class="text" />
		<input type="submit" value="Search" />
	</form>

	<div id="navigation">
		<ul>
			<li>
				<a href="index.html" title="Overview"><span>Overview</span></a>
			</li>
			<li>
				<a href="class-SpiderBot.html" title="Summary of SpiderBot"><span>Class</span></a>
			</li>
		</ul>
		<ul>
			<li>
				<a href="tree.html" title="Tree view of classes, interfaces, traits and exceptions"><span>Tree</span></a>
			</li>
		</ul>
		<ul>
		</ul>
	</div>

<pre><code><span id="1" class="l"><a class="l" href="#1">  1: </a><span class="xlang">&lt;?php</span>
</span><span id="2" class="l"><a class="l" href="#2">  2: </a>
</span><span id="3" class="l"><a class="l" href="#3">  3: </a><span class="php-comment">/*
</span></span><span id="4" class="l"><a class="l" href="#4">  4: </a><span class="php-comment"> * Crawler.php
</span></span><span id="5" class="l"><a class="l" href="#5">  5: </a><span class="php-comment"> * 
</span></span><span id="6" class="l"><a class="l" href="#6">  6: </a><span class="php-comment"> * Copyright 2013 Kaushal L R &lt;exp10r3r@vyadhamarg.info&gt;
</span></span><span id="7" class="l"><a class="l" href="#7">  7: </a><span class="php-comment"> * 
</span></span><span id="8" class="l"><a class="l" href="#8">  8: </a><span class="php-comment"> * This program is free software; you can redistribute it and/or modify
</span></span><span id="9" class="l"><a class="l" href="#9">  9: </a><span class="php-comment"> * it under the terms of the GNU General Public License as published by
</span></span><span id="10" class="l"><a class="l" href="#10"> 10: </a><span class="php-comment"> * the Free Software Foundation; either version 2 of the License, or
</span></span><span id="11" class="l"><a class="l" href="#11"> 11: </a><span class="php-comment"> * (at your option) any later version.
</span></span><span id="12" class="l"><a class="l" href="#12"> 12: </a><span class="php-comment"> * 
</span></span><span id="13" class="l"><a class="l" href="#13"> 13: </a><span class="php-comment"> * This program is distributed in the hope that it will be useful,
</span></span><span id="14" class="l"><a class="l" href="#14"> 14: </a><span class="php-comment"> * but WITHOUT ANY WARRANTY; without even the implied warranty of
</span></span><span id="15" class="l"><a class="l" href="#15"> 15: </a><span class="php-comment"> * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
</span></span><span id="16" class="l"><a class="l" href="#16"> 16: </a><span class="php-comment"> * GNU General Public License for more details.
</span></span><span id="17" class="l"><a class="l" href="#17"> 17: </a><span class="php-comment"> * 
</span></span><span id="18" class="l"><a class="l" href="#18"> 18: </a><span class="php-comment"> * You should have received a copy of the GNU General Public License
</span></span><span id="19" class="l"><a class="l" href="#19"> 19: </a><span class="php-comment"> * along with this program; if not, write to the Free Software
</span></span><span id="20" class="l"><a class="l" href="#20"> 20: </a><span class="php-comment"> * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
</span></span><span id="21" class="l"><a class="l" href="#21"> 21: </a><span class="php-comment"> * MA 02110-1301, USA.
</span></span><span id="22" class="l"><a class="l" href="#22"> 22: </a><span class="php-comment"> * 
</span></span><span id="23" class="l"><a class="l" href="#23"> 23: </a><span class="php-comment"> * 
</span></span><span id="24" class="l"><a class="l" href="#24"> 24: </a><span class="php-comment"> */</span>
</span><span id="25" class="l"><a class="l" href="#25"> 25: </a>
</span><span id="26" class="l"><a class="l" href="#26"> 26: </a><span class="php-comment">//It may take a while to crawl a site, hence a time-limit has to be set.</span>
</span><span id="27" class="l"><a class="l" href="#27"> 27: </a><span class="php-comment">//This file will run only for this set time limit</span>
</span><span id="28" class="l"><a class="l" href="#28"> 28: </a><span class="php-keyword2">set_time_limit</span>(<span class="php-num">10000</span>);
</span><span id="29" class="l"><a class="l" href="#29"> 29: </a>
</span><span id="30" class="l"><a class="l" href="#30"> 30: </a><span class="php-comment">// Include the PHPCrawl main class</span>
</span><span id="31" class="l"><a class="l" href="#31"> 31: </a><span class="php-keyword1">require_once</span> <span class="php-quote">&quot;../lib/nusoap/lib/nusoap.php&quot;</span>;
</span><span id="32" class="l"><a class="l" href="#32"> 32: </a><span class="php-keyword1">include</span>(<span class="php-quote">&quot;../lib/PHPCrawl/libs/PHPCrawler.class.php&quot;</span>);
</span><span id="33" class="l"><a class="l" href="#33"> 33: </a><span class="php-keyword1">require_once</span> <span class="php-quote">'../lib/simple_html_dom.php'</span>;
</span><span id="34" class="l"><a class="l" href="#34"> 34: </a>
</span><span id="35" class="l"><a class="l" href="#35"> 35: </a><span class="php-comment">/**
</span></span><span id="36" class="l"><a class="l" href="#36"> 36: </a><span class="php-comment"> * Configuration for: Error reporting
</span></span><span id="37" class="l"><a class="l" href="#37"> 37: </a><span class="php-comment"> * Useful to show every little problem during development, but only show hard errors in production
</span></span><span id="38" class="l"><a class="l" href="#38"> 38: </a><span class="php-comment"> */</span>
</span><span id="39" class="l"><a class="l" href="#39"> 39: </a><span class="php-comment">//error_reporting(E_ALL);</span>
</span><span id="40" class="l"><a class="l" href="#40"> 40: </a><span class="php-comment">//ini_set(&quot;display_errors&quot;, 1);</span>
</span><span id="41" class="l"><a class="l" href="#41"> 41: </a>
</span><span id="42" class="l"><a class="l" href="#42"> 42: </a><span class="php-comment">/**
</span></span><span id="43" class="l"><a class="l" href="#43"> 43: </a><span class="php-comment"> * This Class extends the PHPCrawler class and has been implemented to run over Web Services.
</span></span><span id="44" class="l"><a class="l" href="#44"> 44: </a><span class="php-comment"> */</span>
</span><span id="45" class="l"><a class="l" href="#45"> 45: </a><span class="php-keyword1">class</span> <a id="SpiderBot" href="#SpiderBot">SpiderBot</a> <span class="php-keyword1">extends</span> PHPCrawler {
</span><span id="46" class="l"><a class="l" href="#46"> 46: </a>
</span><span id="47" class="l"><a class="l" href="#47"> 47: </a>    <span class="php-comment">/**
</span></span><span id="48" class="l"><a class="l" href="#48"> 48: </a><span class="php-comment">     * This function will be called over every unique URL that PHPCrawl finds out.
</span></span><span id="49" class="l"><a class="l" href="#49"> 49: </a><span class="php-comment">     * It will then get the Data from this URL and pass This Data and URL address to our Indexer for Indexing.
</span></span><span id="50" class="l"><a class="l" href="#50"> 50: </a><span class="php-comment">     * @param PHPCrawlerDocumentInfo $DocInfo The Document Object with all other necessary attributes
</span></span><span id="51" class="l"><a class="l" href="#51"> 51: </a><span class="php-comment">     */</span>
</span><span id="52" class="l"><a class="l" href="#52"> 52: </a>    <span class="php-keyword1">function</span> <a id="_handleDocumentInfo" href="#_handleDocumentInfo">handleDocumentInfo</a>(PHPCrawlerDocumentInfo <span class="php-var">$DocInfo</span>) {
</span><span id="53" class="l"><a class="l" href="#53"> 53: </a>        <span class="php-var">$client</span> = <span class="php-keyword1">new</span> nusoap_client(<span class="php-quote">&quot;http://spiderbot.local/WebServices/IndexerService.php?wsdl&quot;</span>,<span class="php-keyword1">true</span>);
</span><span id="54" class="l"><a class="l" href="#54"> 54: </a>        <span class="php-var">$error</span> = <span class="php-var">$client</span>-&gt;getError();
</span><span id="55" class="l"><a class="l" href="#55"> 55: </a>        <span class="php-keyword1">if</span> (<span class="php-var">$error</span>) {
</span><span id="56" class="l"><a class="l" href="#56"> 56: </a>            <span class="php-keyword1">echo</span> <span class="php-quote">&quot;&lt;h2&gt;Constructor error&lt;/h2&gt;&lt;pre&gt;&quot;</span> . <span class="php-var">$error</span> . <span class="php-quote">&quot;&lt;/pre&gt;&quot;</span>;
</span><span id="57" class="l"><a class="l" href="#57"> 57: </a>        }
</span><span id="58" class="l"><a class="l" href="#58"> 58: </a>        <span class="php-comment">// Just detect linebreak for output (&quot;\n&quot; in CLI-mode, otherwise &quot;&lt;br&gt;&quot;). </span>
</span><span id="59" class="l"><a class="l" href="#59"> 59: </a>        <span class="php-keyword1">if</span> (PHP_SAPI == <span class="php-quote">&quot;cli&quot;</span>) {
</span><span id="60" class="l"><a class="l" href="#60"> 60: </a>            <span class="php-var">$lb</span> = <span class="php-quote">&quot;\n&quot;</span>;
</span><span id="61" class="l"><a class="l" href="#61"> 61: </a>        } <span class="php-keyword1">else</span> {
</span><span id="62" class="l"><a class="l" href="#62"> 62: </a>            <span class="php-var">$lb</span> = <span class="php-quote">&quot;&lt;br /&gt;&quot;</span>;
</span><span id="63" class="l"><a class="l" href="#63"> 63: </a>        }
</span><span id="64" class="l"><a class="l" href="#64"> 64: </a>        <span class="php-comment">// Print the URL and the HTTP-status-Code </span>
</span><span id="65" class="l"><a class="l" href="#65"> 65: </a>        <span class="php-keyword1">echo</span> <span class="php-quote">&quot;Page requested: &quot;</span> . <span class="php-var">$DocInfo</span>-&gt;url . <span class="php-quote">&quot; (&quot;</span> . <span class="php-var">$DocInfo</span>-&gt;http_status_code . <span class="php-quote">&quot;)&quot;</span> . <span class="php-var">$lb</span>;
</span><span id="66" class="l"><a class="l" href="#66"> 66: </a>
</span><span id="67" class="l"><a class="l" href="#67"> 67: </a>        <span class="php-comment">// Print the refering URL </span>
</span><span id="68" class="l"><a class="l" href="#68"> 68: </a>        <span class="php-keyword1">echo</span> <span class="php-quote">&quot;Referer-page: &quot;</span> . <span class="php-var">$DocInfo</span>-&gt;referer_url . <span class="php-var">$lb</span>;
</span><span id="69" class="l"><a class="l" href="#69"> 69: </a>
</span><span id="70" class="l"><a class="l" href="#70"> 70: </a>        <span class="php-comment">// Print if the content of the document was be recieved or not </span>
</span><span id="71" class="l"><a class="l" href="#71"> 71: </a>        <span class="php-keyword1">if</span> (<span class="php-var">$DocInfo</span>-&gt;received == <span class="php-keyword1">true</span>) {
</span><span id="72" class="l"><a class="l" href="#72"> 72: </a>            <span class="php-keyword1">echo</span> <span class="php-quote">&quot;Content received: &quot;</span> . <span class="php-var">$DocInfo</span>-&gt;bytes_received . <span class="php-quote">&quot; bytes&quot;</span> . <span class="php-var">$lb</span>;
</span><span id="73" class="l"><a class="l" href="#73"> 73: </a>            <span class="php-comment">//echo 'Data' . file_get_html($DocInfo-&gt;url)-&gt;plaintext;</span>
</span><span id="74" class="l"><a class="l" href="#74"> 74: </a>            <span class="php-keyword1">echo</span> <span class="php-quote">'&lt;b&gt; Calling indexer&lt;/b&gt;'</span>;
</span><span id="75" class="l"><a class="l" href="#75"> 75: </a>            <span class="php-var">$result</span> = <span class="php-var">$client</span>-&gt;call(<span class="php-quote">&quot;indexDocument&quot;</span>, <span class="php-keyword1">array</span>(<span class="php-quote">&quot;documentData&quot;</span> =&gt; file_get_html(<span class="php-var">$DocInfo</span>-&gt;url)-&gt;plaintext, <span class="php-quote">&quot;url&quot;</span> =&gt; <span class="php-var">$DocInfo</span>-&gt;url));
</span><span id="76" class="l"><a class="l" href="#76"> 76: </a>            <span class="php-keyword1">if</span> (<span class="php-var">$client</span>-&gt;fault) {
</span><span id="77" class="l"><a class="l" href="#77"> 77: </a>                <span class="php-keyword1">echo</span> <span class="php-quote">'&lt;br/&gt;&lt;b&gt; Fault occured&lt;/b&gt;'</span>;
</span><span id="78" class="l"><a class="l" href="#78"> 78: </a>            } <span class="php-keyword1">else</span> {
</span><span id="79" class="l"><a class="l" href="#79"> 79: </a>                <span class="php-var">$err</span> = <span class="php-var">$client</span>-&gt;getError();
</span><span id="80" class="l"><a class="l" href="#80"> 80: </a>                <span class="php-keyword1">if</span> (<span class="php-var">$err</span>) {
</span><span id="81" class="l"><a class="l" href="#81"> 81: </a>                    <span class="php-comment">// Display the error</span>
</span><span id="82" class="l"><a class="l" href="#82"> 82: </a>                    <span class="php-keyword1">echo</span> <span class="php-quote">'&lt;h2&gt;Error&lt;/h2&gt;&lt;pre&gt;'</span> . <span class="php-var">$err</span> . <span class="php-quote">'&lt;/pre&gt;'</span>;
</span><span id="83" class="l"><a class="l" href="#83"> 83: </a>                } <span class="php-keyword1">else</span> {
</span><span id="84" class="l"><a class="l" href="#84"> 84: </a>                    <span class="php-comment">// Display the result</span>
</span><span id="85" class="l"><a class="l" href="#85"> 85: </a>                    <span class="php-keyword1">echo</span> <span class="php-quote">'&lt;h2&gt;Result&lt;/h2&gt;&lt;pre&gt;'</span>;
</span><span id="86" class="l"><a class="l" href="#86"> 86: </a>                    <span class="php-keyword2">print_r</span>(<span class="php-var">$result</span>);
</span><span id="87" class="l"><a class="l" href="#87"> 87: </a>                    <span class="php-keyword1">echo</span> <span class="php-quote">'&lt;/pre&gt;'</span>;
</span><span id="88" class="l"><a class="l" href="#88"> 88: </a>                }
</span><span id="89" class="l"><a class="l" href="#89"> 89: </a>            }
</span><span id="90" class="l"><a class="l" href="#90"> 90: </a>        } <span class="php-keyword1">else</span> {
</span><span id="91" class="l"><a class="l" href="#91"> 91: </a>            <span class="php-keyword1">echo</span> <span class="php-quote">&quot;Content not received&quot;</span> . <span class="php-var">$lb</span>;
</span><span id="92" class="l"><a class="l" href="#92"> 92: </a>        }
</span><span id="93" class="l"><a class="l" href="#93"> 93: </a>
</span><span id="94" class="l"><a class="l" href="#94"> 94: </a>        <span class="php-comment">// Now you should do something with the content of the actual </span>
</span><span id="95" class="l"><a class="l" href="#95"> 95: </a>        <span class="php-comment">// received page or file ($DocInfo-&gt;source), we skip it in this example  </span>
</span><span id="96" class="l"><a class="l" href="#96"> 96: </a>
</span><span id="97" class="l"><a class="l" href="#97"> 97: </a>        <span class="php-keyword1">echo</span> <span class="php-var">$lb</span>;
</span><span id="98" class="l"><a class="l" href="#98"> 98: </a>
</span><span id="99" class="l"><a class="l" href="#99"> 99: </a>        <span class="php-keyword2">flush</span>();
</span><span id="100" class="l"><a class="l" href="#100">100: </a>    }
</span><span id="101" class="l"><a class="l" href="#101">101: </a>
</span><span id="102" class="l"><a class="l" href="#102">102: </a>}
</span><span id="103" class="l"><a class="l" href="#103">103: </a>
</span><span id="104" class="l"><a class="l" href="#104">104: </a><span class="php-comment">// Now, create a instance of your class, define the behaviour </span>
</span><span id="105" class="l"><a class="l" href="#105">105: </a><span class="php-comment">// of the crawler (see class-reference for more options and details) </span>
</span><span id="106" class="l"><a class="l" href="#106">106: </a><span class="php-comment">// and start the crawling-process.  </span>
</span><span id="107" class="l"><a class="l" href="#107">107: </a>
</span><span id="108" class="l"><a class="l" href="#108">108: </a><span class="php-var">$crawler</span> = <span class="php-keyword1">new</span> SpiderBot();
</span><span id="109" class="l"><a class="l" href="#109">109: </a>
</span><span id="110" class="l"><a class="l" href="#110">110: </a><span class="php-comment">// URL to crawl </span>
</span><span id="111" class="l"><a class="l" href="#111">111: </a><span class="php-var">$crawler</span>-&gt;setURL(<span class="php-var">$_POST</span>[<span class="php-quote">'URL'</span>]);
</span><span id="112" class="l"><a class="l" href="#112">112: </a>
</span><span id="113" class="l"><a class="l" href="#113">113: </a><span class="php-comment">// Only receive content of files with content-type &quot;text/html&quot; </span>
</span><span id="114" class="l"><a class="l" href="#114">114: </a><span class="php-var">$crawler</span>-&gt;addContentTypeReceiveRule(<span class="php-quote">&quot;#text/html#&quot;</span>);
</span><span id="115" class="l"><a class="l" href="#115">115: </a>
</span><span id="116" class="l"><a class="l" href="#116">116: </a><span class="php-comment">// Ignore links to pictures, dont even request pictures </span>
</span><span id="117" class="l"><a class="l" href="#117">117: </a><span class="php-var">$crawler</span>-&gt;addURLFilterRule(<span class="php-quote">&quot;#\.(jpg|jpeg|gif|png)</span><span class="php-var">$</span><span class="php-quote"># i&quot;</span>);
</span><span id="118" class="l"><a class="l" href="#118">118: </a>
</span><span id="119" class="l"><a class="l" href="#119">119: </a><span class="php-comment">// Store and send cookie-data like a browser does </span>
</span><span id="120" class="l"><a class="l" href="#120">120: </a><span class="php-var">$crawler</span>-&gt;enableCookieHandling(<span class="php-keyword1">true</span>);
</span><span id="121" class="l"><a class="l" href="#121">121: </a>
</span><span id="122" class="l"><a class="l" href="#122">122: </a><span class="php-comment">// Set the traffic-limit to 1 MB (in bytes, </span>
</span><span id="123" class="l"><a class="l" href="#123">123: </a><span class="php-comment">// for testing we dont want to &quot;suck&quot; the whole site) </span>
</span><span id="124" class="l"><a class="l" href="#124">124: </a><span class="php-var">$crawler</span>-&gt;setTrafficLimit(<span class="php-num">500</span> * <span class="php-num">128</span>);
</span><span id="125" class="l"><a class="l" href="#125">125: </a>
</span><span id="126" class="l"><a class="l" href="#126">126: </a><span class="php-keyword1">echo</span> <span class="php-quote">&quot;&lt;b&gt;Indexing : &quot;</span>. <span class="php-var">$_POST</span>[<span class="php-quote">'URL'</span>] . <span class="php-quote">&quot;&lt;/b&gt;&lt;br/&gt;&quot;</span>;
</span><span id="127" class="l"><a class="l" href="#127">127: </a>
</span><span id="128" class="l"><a class="l" href="#128">128: </a><span class="php-comment">// Thats enough, now here we go </span>
</span><span id="129" class="l"><a class="l" href="#129">129: </a><span class="php-var">$crawler</span>-&gt;go();
</span><span id="130" class="l"><a class="l" href="#130">130: </a>
</span><span id="131" class="l"><a class="l" href="#131">131: </a><span class="php-comment">// At the end, after the process is finished, we print a short </span>
</span><span id="132" class="l"><a class="l" href="#132">132: </a><span class="php-comment">// report (see method getProcessReport() for more information) </span>
</span><span id="133" class="l"><a class="l" href="#133">133: </a><span class="php-var">$report</span> = <span class="php-var">$crawler</span>-&gt;getProcessReport();
</span><span id="134" class="l"><a class="l" href="#134">134: </a>
</span><span id="135" class="l"><a class="l" href="#135">135: </a><span class="php-keyword1">if</span> (PHP_SAPI == <span class="php-quote">&quot;cli&quot;</span>) {
</span><span id="136" class="l"><a class="l" href="#136">136: </a>    <span class="php-var">$lb</span> = <span class="php-quote">&quot;\n&quot;</span>;
</span><span id="137" class="l"><a class="l" href="#137">137: </a>} <span class="php-keyword1">else</span> {
</span><span id="138" class="l"><a class="l" href="#138">138: </a>    <span class="php-var">$lb</span> = <span class="php-quote">&quot;&lt;br /&gt;&quot;</span>;
</span><span id="139" class="l"><a class="l" href="#139">139: </a>}
</span><span id="140" class="l"><a class="l" href="#140">140: </a>
</span><span id="141" class="l"><a class="l" href="#141">141: </a><span class="php-keyword1">echo</span> <span class="php-quote">&quot;Summary:&quot;</span> . <span class="php-var">$lb</span>;
</span><span id="142" class="l"><a class="l" href="#142">142: </a><span class="php-keyword1">echo</span> <span class="php-quote">&quot;Links followed: &quot;</span> . <span class="php-var">$report</span>-&gt;links_followed . <span class="php-var">$lb</span>;
</span><span id="143" class="l"><a class="l" href="#143">143: </a><span class="php-keyword1">echo</span> <span class="php-quote">&quot;Documents received: &quot;</span> . <span class="php-var">$report</span>-&gt;files_received . <span class="php-var">$lb</span>;
</span><span id="144" class="l"><a class="l" href="#144">144: </a><span class="php-keyword1">echo</span> <span class="php-quote">&quot;Bytes received: &quot;</span> . <span class="php-var">$report</span>-&gt;bytes_received . <span class="php-quote">&quot; bytes&quot;</span> . <span class="php-var">$lb</span>;
</span><span id="145" class="l"><a class="l" href="#145">145: </a><span class="php-keyword1">echo</span> <span class="php-quote">&quot;Process runtime: &quot;</span> . <span class="php-var">$report</span>-&gt;process_runtime . <span class="php-quote">&quot; sec&quot;</span> . <span class="php-var">$lb</span>;
</span><span id="146" class="l"><a class="l" href="#146">146: </a></span></code></pre>

	<div id="footer">
		 API documentation generated by <a href="http://apigen.org">ApiGen 2.8.0</a>
	</div>
</div>
</div>
</body>
</html>
